#include <iostream>
#include <stdio.h>
#include <curl/curl.h>
#include <curl/types.h>
#include <curl/easy.h>

#include "URL.h"

#include "DownloadParser.h"
#include "../classes_dados/Repository.h"
#include "../classes_logger/DebugLogger.h"

using namespace std;

int DownloadParser::numSitesUrl = 0;


string DownloadParser::getHeader(){
	try{
		curl = curl_easy_init();
		if(curl){
			curl_easy_setopt(curl, CURLOPT_URL, site_url.c_str());
			curl_config();
			curl_easy_cleanup(curl);
		}
	}catch(...){
		cout << "Vixi" << endl;
	}
	return header_site;
}


string DownloadParser::getRobot(){
	try{
		//Teste de Tratamento	
		if( (site_url.size() > 300) || (site_url.size() <= 0) ) return ""; 
		string site_robot = site_url + "robots.txt";
		
		curl = curl_easy_init();
		if(curl){
			curl_easy_setopt(curl, CURLOPT_URL, site_robot.c_str());
			curl_config();	
			curl_easy_cleanup(curl);
		}
		// Tratamento de erro simplista
		if(res_code == 200) return data_site;
		else                return "";
	}catch(...){
		cout << "Vixi" << endl;
		return " ";
	}
}

string DownloadParser::getPage(){
	try{	
		//Teste de Tratamento	
		if( (site_url.size() > 300) || (site_url.size() <= 0) ) return ""; 
		
		curl = curl_easy_init();
		if(curl){
			curl_easy_setopt(curl, CURLOPT_URL, site_url.c_str());
			curl_config();
			curl_easy_cleanup(curl);
		}
		// Tratamento de erro simplista
		if(res_code == 200) return data_site;
		else                return "";
	}catch(...){
		cout << "Vixi" << endl;
		return "";
	}	
}



size_t DownloadParser::WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *userp){
	size_t realsize = size * nmemb;
	try{
		
		if( (ptr == NULL) || (nmemb==0) )  return realsize;
		// char *teste = (char *) ptr;
		string *data_str = (string *) userp;
		// for(uint i=0; i < nmemb; i++) *data_str += teste[i];
		
		data_str->append((char *)ptr, nmemb);  // parce melhor...

	}catch(...){
		cout << "Vixi Especial..." << endl;
	} 
	return realsize;
}

size_t DownloadParser::headerCallback(void *ptr, size_t size, size_t nmemb, void *userp){
	size_t realsize = size * nmemb;
	try{
		
		// char *teste = (char *) ptr;
		// string *data_str = (string *) userp;
		// for(uint i=0; i < nmemb; i++) *data_str += teste[i];
		
	}catch(...){
		cout << "Vixi Especial..." << endl;
	}
	return realsize;
}



list<URL>  DownloadParser::parser(string buffer_page){
	
	unsigned int count_link = 0;	
	list<string> buffer_links;
	list<URL> buffer_urls;
	
	std::string::size_type pos_final;
	std::string::size_type pos_atual;
	
	//acha posicao final do arquivo	url_rejeita = "()'!";
	pos_final = buffer_page.size();
	pos_atual = buffer_page.find(s_href, 0);
	do{
		std::string::size_type pos_aspa_begin;
		std::string::size_type pos_aspa_end;
		if(pos_atual >= pos_final) continue;
		//pega o link, se existe
		pos_aspa_begin = buffer_page.find("\"", pos_atual);
		pos_aspa_end = buffer_page.find("\"",pos_aspa_begin+1);
		buffer_links.push_back(buffer_page.substr(pos_aspa_begin+1, pos_aspa_end-pos_aspa_begin-1));
		pos_atual = buffer_page.find(s_href, pos_atual+1);
		count_link++;
	}while(pos_atual < pos_final);
	
	// Aplica regras para extrair falsos links
	rejeita_links(buffer_links);
	// Normalize
	buffer_urls = eliminaManjados( normalisa.normalize(buffer_links) );  // canoniza URLs
	
	return buffer_urls;
}



list<URL> DownloadParser::getNewURLs(void){

	buffer_links.clear();
	buffer_page.clear();
	
	//carrega site
	buffer_page = getPage();
	
	//grava site
	disco->store(site_url, buffer_page);
	
	return parser(buffer_page);
}



list<URL> DownloadParser::eliminaManjados(list<URL> linkList) {
	list<string> listaDeFormatosManjados;
	list<URL> listaFiltrada;
	try{	
		
		listaDeFormatosManjados.push_front(".jpg");
		listaDeFormatosManjados.push_front(".pdf");
		listaDeFormatosManjados.push_front(".doc");
		listaDeFormatosManjados.push_front(".pps");
		listaDeFormatosManjados.push_front(".gif");
		listaDeFormatosManjados.push_front(".bmp");
		listaDeFormatosManjados.push_front(".swf");
		listaDeFormatosManjados.push_front(".mp3");
		listaDeFormatosManjados.push_front(".avi");
		listaDeFormatosManjados.push_front(".ico");
		listaDeFormatosManjados.push_front(".mpeg");
		listaDeFormatosManjados.push_front(".mpg");
		listaDeFormatosManjados.push_front(".jpeg");
		listaDeFormatosManjados.push_front(".png");
		listaDeFormatosManjados.push_front(".cdr");
		listaDeFormatosManjados.push_front(".wav");
		listaDeFormatosManjados.push_front(".css");
		
		list<string>::iterator stringIt;
		list<URL>::iterator urlIt;
		
		string path;
		for (urlIt=linkList.begin(); urlIt!=linkList.end(); urlIt++) {
			listaFiltrada.push_front(*urlIt);
			for (stringIt=listaDeFormatosManjados.begin(); stringIt!=listaDeFormatosManjados.end(); stringIt++) {
				path = urlIt->get_path();
				if ( path.rfind(*stringIt) != string::npos) {
					listaFiltrada.pop_front();
					break;
				}
			}
		}
		
	}catch(...){
		cout << "Vixi..." << endl;
	}
	return listaFiltrada;
}


string  DownloadParser::achaDominio (string str) {
	try{
		uint index = str.find("://");
		if (index != string::npos) { 
			str = str.substr( index+3, str.size() );
		}
		uint slashIndex = str.find("/");
		if (slashIndex != string::npos) 
			return str.substr(0, slashIndex);
	}catch(...){
		cout << "Vixi..." << endl;
	}
	return str;
}



void DownloadParser::rejeita_links(std::list <std::string> &lista_in){
//	DebugLogger dbug_rejeita_links("dbug_rejeita_links");
	std::string temp_url;
	string url_rejeita;
	int buffer_links_size = (int) lista_in.size();
	try{		
		
		url_rejeita = "()'!@{}|\\^[]`<>";
		//referencia para links relativos
		uint pos_barra = pagina_at.find_last_of("/"); 
		if( pos_barra != std::string::npos) pagina_at.erase(pos_barra);
		//Grava em arquivo a lista
		for( int i=0; i<buffer_links_size; i++){
			temp_url = lista_in.front();
			if( temp_url.find_first_of(url_rejeita) == std::string::npos
				&& temp_url.find_first_of("./") != std::string::npos ){
					// Tira carecteres indesejados
					for(unsigned int t=0; t < temp_url.size(); t++)
						if(temp_url.at(t) == ' ' || temp_url.at(t) == '\n'){
							temp_url.erase(t,1); // tira caractere
							t--;           // volta uma pos.
						}
					//resolve links relativos
					if (temp_url.at(0) == '/') {
						temp_url = achaDominio(pagina_at) + temp_url;
					}
					
					if(temp_url.at(0) == '.'){
						for(uint t=0; t<=temp_url.size(); t++)
							if(temp_url.at(0) == '/' || temp_url.at(0) == '.'){
								if(temp_url.at(0) == '.' && temp_url.at(1) == '.'){
									temp_url.erase(0,2);
									pos_barra = pagina_at.find_last_of("/"); 
									if( pos_barra != std::string::npos){
										if( pagina_at.find_last_of("/") > pagina_at.find_last_of(".")  
											&& pagina_at.find_last_of("/") != std::string::npos
											&& pagina_at.find_last_of("/") != std::string::npos) pagina_at.erase(pos_barra); //retorna dir se puder
									}
								}
								temp_url.erase(0,1);
								if(temp_url.find_first_of("./") == std::string::npos){
									temp_url = pagina_at + '/' + temp_url; 
									break;
								}
							}else{
								temp_url = pagina_at + '/' + temp_url;
								break;
							}
					}else if(   temp_url.find("://") == std::string::npos &&
							  ( temp_url.find_first_of("/") < temp_url.find_first_of(".") ||
							    temp_url.find_first_of("/") == std::string::npos )){
						temp_url = pagina_at + '/' + temp_url;
					}
					lista_in.push_back(temp_url);
					lista_in.pop_front();
			}
			
			else{
				lista_in.pop_front();
			}
		}
		
	}catch(...){ // Gambi
//		dbug_rejeita_links.enable(DebugLogger::info + DebugLogger::alert + DebugLogger::error);
//		dbug_rejeita_links.errorMsg(lista_in.front());
	}
}



int curl_trace(CURL *handle, curl_infotype type, unsigned char *data, size_t size, void *userp){
	
//  struct data *config = (struct data *)userp;
//  (void) handle; /* prevent compiler warning */
//  const char *text;	
//  switch (type) {
//  case CURLINFO_TEXT:
//	  //cout << "- Info: "<< data << endl;
//  default: /* in case a new one is introduced to shock us */
//	  return 0;
//  case CURLINFO_HEADER_OUT:
//    text = "=> Send header";
//    break;
//  case CURLINFO_DATA_OUT:
//    text = "=> Send data";
//    break;
//  case CURLINFO_SSL_DATA_OUT:
//    text = "=> Send SSL data";
//    break;
//  case CURLINFO_HEADER_IN:
//    text = "<= Recv header";
//    break;
//  case CURLINFO_DATA_IN:
//    text = "<= Recv data";
//    break;
//  case CURLINFO_SSL_DATA_IN:
//    text = "<= Recv SSL data";
//    break;
//  }
//cout << "All Data: "<< data << endl;
//cout << text << endl;
//  dump(text, stderr, data, size, config->trace_ascii);
  return 0;
}


int progress_callback(void *userp, double dltotal, double dlnow, double ultotal, double ulnow){
//	cout << "dltotal: " << dltotal << endl; //download total
//	cout << "dlnow: " << dltotal << endl;   //download now
//	cout << "ultotal: " << dltotal << endl; //upload total
//	cout << "ulnow: " << dltotal << endl;   //upload now
	return 0;
}



int DownloadParser::curl_config(void) {
	 CURLcode res;
	 struct data config;
	 config.trace_ascii = 1; /* enable ascii tracing */
	 cleanBuffer();
	 if(curl) {
		 curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, curl_trace);
	     curl_easy_setopt(curl, CURLOPT_DEBUGDATA, &config);            // dado generico do usuario
	     curl_easy_setopt(curl, CURLOPT_VERBOSE, 0);
	     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
	     curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data_site);
	     curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback );
	     curl_easy_setopt(curl, CURLOPT_HEADERDATA,&header_site);
	   
	   //Tamanho maximo para arquivo (em bytes)
	   curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, 300*1024); //limita 300 kbytes
	   
	   //Limite de velocidade (em bytes por segundo)
       //    CURLOPT_LOW_SPEED_LIMIT
	       
	   //Redirecionamentos
	   curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0);
//	   curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 3); //maximo Redirecionamento. se Follow true
	                   
	   curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5);
	   
	   //deve setar para TRUE se em multi-Thread
	   curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
	   
	//    curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0");
	//    curl_easy_setopt(curl, CURLOPT_HEADER, 1);
	//    curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
	//    curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback);
	   
	//    http://www.pudim.com.br/robots.txt
	//   --> MC: entre 5 e 6 milhao
	//   --> Baggio: entre 1 e 2 milhao
	//    http://www.ogrilofeliz.com.br/robots.txt
	   
	   
	   res = curl_easy_perform(curl);
	   res = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &res_code);    
//	   cout << curl_easy_strerror(res);  // imprime erro bonitinho em char*
	 }
	 return 0;
}



// Referencia: http://www.ietf.org/rfc/rfc2396.txt
//             http://gbiv.com/protocols/uri/rfc/rfc3986.html
//   reserved    = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | ","
//   unreserved  = alphanum | mark
//   mark        = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"

// Excluidos:
//  control     = <US-ASCII coded characters 00-1F and 7F hexadecimal>
//  space       = <US-ASCII coded character 20 hexadecimal>
//  delims      = "<" | ">" | "#" | "%" | <">
//  unwise      = "{" | "}" | "|" | "\" | "^" | "[" | "]" | "`""

//
//CURLOPT_RANGE 
//
//Pass a char * as parameter, which should contain the specified range you want. It should be in the format "X-Y", where X or Y may be left out. HTTP transfers also support several intervals, separated with commas as in "X-Y,N-M". Using this kind of multiple intervals will cause the HTTP server to send the response document in pieces (using standard MIME separation techniques). Pass a NULL to this option to disable the use of ranges. 
//
//Ranges work on HTTP, FTP and FILE (since 7.18.0) transfers only.

//CURLOPT_TIMEOUT 
//
//Pass a long as parameter containing the maximum time in seconds that you allow the libcurl transfer operation to take. Normally, name lookups can take a considerable time and limiting operations to less than a few minutes risk aborting perfectly normal operations. This option will cause curl to use the SIGALRM to enable time-outing system calls. 
//
//In unix-like systems, this might cause signals to be used unless CURLOPT_NOSIGNAL is set. 
//
//CURLOPT_TIMEOUT_MS 
//
//Like CURLOPT_TIMEOUT but takes number of milliseconds instead. If libcurl is built to use the standard system name resolver, that part will still use full-second resolution for timeouts. (Added in 7.16.2)

